#!/usr/bin/env python
# encoding: utf-8
"""
@author: youfeng
@email: youfeng243@163.com
@license: Apache Licence 
@file: start_parse_db.py
@time: 2016/12/22 15:37
"""
import random
import sys
import time
from multiprocessing import Pool

from common import util
from common.config_parser import ConfigParser
from common.generator import create_crawl_object
from common.mongo import MongDb
from config.conf import mongo_db_source
from logger import Gsxtlogger

log = Gsxtlogger(util.get_pid_log_name('court_multiprocess_crawler')).get_logger()

# 连接mongodb
source_db = MongDb(mongo_db_source['host'], mongo_db_source['port'], mongo_db_source['db'],
                   mongo_db_source['username'], mongo_db_source['password'], log=log)


def run_task(config, item_list):
    from multiprocessing.dummy import Pool as ThreadPool
    worker = create_crawl_object(config, '')

    thread_num = int(config.get('thread_num', '14'))
    worker.log.info('线程数目为: thread_num = {num}'.format(num=thread_num))

    try:
        # 创建线程池
        pool = ThreadPool(processes=thread_num)
        try:
            for item in item_list:
                pool.apply_async(worker.query_task, args=(item,))
        except Exception as e:
            worker.log.exception(e)

        pool.close()
        pool.join()
    except Exception as e:
        worker.log.exception(e)


def get_db_data(config):
    source_table = config.get('source_table', '')
    source_select_param = eval(config.get('source_select_param', 'None'))
    if source_select_param is None:
        raise StandardError('没有筛选条件')

    company_list = []
    start_time = time.time()
    log.info('开始加载数据库种子列表..')
    for item in source_db.select(source_table, source_select_param):
        company_list.append(item)
        if len(company_list) >= 5000000:
            break
    log.info('加载数据库数据个数为: size = {size}'.format(size=len(company_list)))
    end_time = time.time()

    log.info('加载数据起始时间: {st}'.format(st=start_time))
    log.info('加载数据结束时间: {et}'.format(et=end_time))
    log.info('加载数据消耗时间: {t}s'.format(t=end_time - start_time))

    return company_list


def main():
    session_key = 'shixin'
    config_file = 'config/cmb_court.conf'
    p_num = 1
    if len(sys.argv) >= 4:
        session_key = sys.argv[1]
        config_file = sys.argv[2]
        p_num = int(sys.argv[3])

    conf_parse = ConfigParser(config_file)
    config = conf_parse.get_session(session_key)

    cnt = 1
    while True:
        process_num = p_num
        item_list = get_db_data(config)
        length = len(item_list)

        if length <= 0:
            log.info('当前加载数据数目为: {num} 随机休眠300 - 500 秒'.format(num=length))
            time.sleep(random.randint(300, 500))
            continue

        if length <= process_num:
            process_num = 1

        piece = length / process_num
        pool = Pool(processes=process_num)
        log.info('开始第{cnt}轮迭代...'.format(cnt=cnt))
        log.info('当前加载数据数目为: {num}'.format(num=length))
        for i in xrange(process_num):
            if i == process_num - 1:
                pool.apply_async(run_task, args=(config, item_list[i * piece:]))
            else:
                pool.apply_async(run_task, args=(config, item_list[i * piece: (i + 1) * piece]))

        pool.close()
        pool.join()

        log.info('第{cnt}轮迭代结束...'.format(cnt=cnt))
        cnt += 1
        time.sleep(random.randint(300, 500))


if __name__ == '__main__':
    main()
